# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from mongodb_models import steam_infos, brainpop_steam_infos, nasa_steam_infos
from steam_data_collect.func_pack import filter_words, filter_repeats


class SteamDataCollectPipeline(object):
    def process_item(self, item, spider):
        if spider.name=="steam":
            if 'class_title' in item.keys():
                steam_list=steam_infos.objects(url=item['url'])
                if steam_list.count()==0:
                    new_steam=steam_infos(url=item['url'],class_title=item['class_title'],class_desc=item['class_desc'],class_img=item['class_img'],challenge_title=item['challenge_title'],challenge_desc=item['challenge_desc'],
                            objective=item['objective'],background_information=item['background_information'],science_skills=item['science_skills'],technology_skills=item['technology_skills'],
                                engineering_skills=item['engineering_skills'],arts_skills=item['arts_skills'],math_skills=item['math_skills'],directions=item['directions'],materials=item['materials'],
                                wrap_up=item['wrap_up'],extended_picture=item['extended_picture'],achievements_picture=item['achievements_picture'])
                    new_steam.save()
                else:
                    one_steam=steam_list[0]
                    one_steam.class_title=item['class_title']
                    one_steam.class_desc = item['class_desc']
                    one_steam.class_img = item['class_img']
                    one_steam.challenge_title = item['challenge_title']
                    one_steam.challenge_desc = item['challenge_desc']
                    one_steam.objective = item['objective']
                    one_steam.background_information = item['background_information']
                    one_steam.science_skills = item['science_skills']
                    one_steam.technology_skills = item['technology_skills']
                    one_steam.engineering_skills = item['engineering_skills']
                    one_steam.arts_skills = item['arts_skills']
                    one_steam.math_skills = item['math_skills']
                    one_steam.directions = item['directions']
                    one_steam.materials = item['materials']
                    one_steam.wrap_up = item['wrap_up']
                    one_steam.extended_picture = item['extended_picture']
                    one_steam.achievements_picture = item['achievements_picture']
                    one_steam.save()
        return item



class BrainpopSteamDataCollectPipeline(object):
    def process_item(self, item, spider):
        if spider.name=="brainpop_steam":
            if 'class_title' in item.keys():
                item["class_title"] = map(lambda x: filter_words(x),item["class_title"])
                item["class_desc"] = map(lambda x: filter_words(x),item["class_desc"])
                item["challenge_desc"] = map(lambda x: filter_words(x),item["challenge_desc"])
                item["objective"] = map(lambda x: filter_words(x),item["objective"])
                item["directions"] = map(lambda x: filter_words(x),item["directions"])
                item["materials"] = map(lambda x: filter_words(x),item["materials"])
                item["extended_content"] = map(lambda x: filter_words(x),item["extended_content"])

                item["class_title"] = map(lambda x: filter_repeats(x), item["class_title"])
                item["class_desc"] = map(lambda x: filter_repeats(x), item["class_desc"])
                item["challenge_desc"] = map(lambda x: filter_repeats(x), item["challenge_desc"])
                item["objective"] = map(lambda x: filter_repeats(x), item["objective"])
                item["directions"] = map(lambda x: filter_repeats(x), item["directions"])
                item["materials"] = map(lambda x: filter_repeats(x), item["materials"])
                item["extended_content"] = map(lambda x: filter_repeats(x), item["extended_content"])

                item["class_title"].remove("") if "" in item["class_title"] else item["class_title"]
                item["class_desc"].remove("") if "" in item["class_desc"] else item["class_desc"]
                item["challenge_desc"].remove("") if "" in item["challenge_desc"] else item["challenge_desc"]
                item["objective"].remove("") if "" in item["objective"] else item["objective"]
                item["directions"].remove("") if "" in item["directions"] else item["directions"]
                item["materials"].remove("") if "" in item["materials"] else item["materials"]
                item["extended_content"].remove("") if "" in item["extended_content"] else item["extended_content"]


            if 'class_title' in item.keys():
                steam_list=brainpop_steam_infos.objects(url=item['url'])
                if steam_list.count()==0:
                    new_steam=brainpop_steam_infos(url=item['url'],class_title=item['class_title'],class_desc=item['class_desc'],class_img=item['class_img'],challenge_title=item['challenge_title'],challenge_desc=item['challenge_desc'],
                            objective=item['objective'],background_information=item['background_information'],science_skills=item['science_skills'],technology_skills=item['technology_skills'],
                                engineering_skills=item['engineering_skills'],arts_skills=item['arts_skills'],math_skills=item['math_skills'],directions=item['directions'],materials=item['materials'],
                                wrap_up=item['wrap_up'],extended_content=item['extended_content'],extended_picture=item['extended_picture'],achievements_picture=item['achievements_picture'])
                    new_steam.save()
                else:
                    one_steam=steam_list[0]
                    one_steam.class_title=item['class_title']
                    one_steam.class_desc = item['class_desc']
                    one_steam.class_img = item['class_img']
                    one_steam.challenge_title = item['challenge_title']
                    one_steam.challenge_desc = item['challenge_desc']
                    one_steam.objective = item['objective']
                    one_steam.background_information = item['background_information']
                    one_steam.science_skills = item['science_skills']
                    one_steam.technology_skills = item['technology_skills']
                    one_steam.engineering_skills = item['engineering_skills']
                    one_steam.arts_skills = item['arts_skills']
                    one_steam.math_skills = item['math_skills']
                    one_steam.directions = item['directions']
                    one_steam.materials = item['materials']
                    one_steam.wrap_up = item['wrap_up']
                    one_steam.extended_content = item['extended_content']
                    one_steam.extended_picture = item['extended_picture']
                    one_steam.achievements_picture = item['achievements_picture']
                    one_steam.save()
        return item



class NasaSteamDataCollectPipeline(object):
    def process_item(self, item, spider):
        if spider.name=="nasa_steam":
            # item["class_title"] = map(lambda x: filter_words(x),item["class_title"])
            # item["class_desc"] = map(lambda x: filter_words(x),item["class_desc"])
            # item["challenge_desc"] = map(lambda x: filter_words(x),item["challenge_desc"])
            # item["objective"] = map(lambda x: filter_words(x),item["objective"])
            # item["directions"] = map(lambda x: filter_words(x),item["directions"])
            # item["materials"] = map(lambda x: filter_words(x),item["materials"])
            # item["extended_content"] = map(lambda x: filter_words(x),item["extended_content"])
            #
            # item["class_title"] = map(lambda x: filter_repeats(x), item["class_title"])
            # item["class_desc"] = map(lambda x: filter_repeats(x), item["class_desc"])
            # item["challenge_desc"] = map(lambda x: filter_repeats(x), item["challenge_desc"])
            # item["objective"] = map(lambda x: filter_repeats(x), item["objective"])
            # item["directions"] = map(lambda x: filter_repeats(x), item["directions"])
            # item["materials"] = map(lambda x: filter_repeats(x), item["materials"])
            # item["extended_content"] = map(lambda x: filter_repeats(x), item["extended_content"])
            #
            if 'class_title' in item.keys():
                item["class_title"].remove("") if "" in item["class_title"] else item["class_title"]
                item["class_desc"].remove("") if "" in item["class_desc"] else item["class_desc"]
                item["challenge_title"].remove("") if "" in item["challenge_title"] else item["challenge_title"]
                item["challenge_desc"].remove("") if "" in item["challenge_desc"] else item["challenge_desc"]
                item["objective"].remove("") if "" in item["objective"] else item["objective"]
                item["background_information"].remove("") if "" in item["background_information"] else item["background_information"]
                item["directions"].remove("") if "" in item["directions"] else item["directions"]
                item["materials"].remove("") if "" in item["materials"] else item["materials"]
                item["wrap_up"].remove("") if "" in item["wrap_up"] else item["wrap_up"]
                item["extended_content"].remove("") if "" in item["extended_content"] else item["extended_content"]

            if 'class_title' in item.keys():
                steam_list=nasa_steam_infos.objects(url=item['url'])
                if steam_list.count()==0:
                    new_steam=nasa_steam_infos(url=item['url'],class_title=item['class_title'],class_desc=item['class_desc'],class_img=item['class_img'],challenge_title=item['challenge_title'],challenge_desc=item['challenge_desc'],
                            objective=item['objective'],background_information=item['background_information'],science_skills=item['science_skills'],technology_skills=item['technology_skills'],
                                engineering_skills=item['engineering_skills'],arts_skills=item['arts_skills'],math_skills=item['math_skills'],directions=item['directions'],materials=item['materials'],
                                wrap_up=item['wrap_up'],extended_content=item['extended_content'],extended_picture=item['extended_picture'],achievements_picture=item['achievements_picture'])
                    new_steam.save()
                else:
                    one_steam=steam_list[0]
                    one_steam.class_title=item['class_title']
                    one_steam.class_desc = item['class_desc']
                    one_steam.class_img = item['class_img']
                    one_steam.challenge_title = item['challenge_title']
                    one_steam.challenge_desc = item['challenge_desc']
                    one_steam.objective = item['objective']
                    one_steam.background_information = item['background_information']
                    one_steam.science_skills = item['science_skills']
                    one_steam.technology_skills = item['technology_skills']
                    one_steam.engineering_skills = item['engineering_skills']
                    one_steam.arts_skills = item['arts_skills']
                    one_steam.math_skills = item['math_skills']
                    one_steam.directions = item['directions']
                    one_steam.materials = item['materials']
                    one_steam.wrap_up = item['wrap_up']
                    one_steam.extended_content = item['extended_content']
                    one_steam.extended_picture = item['extended_picture']
                    one_steam.achievements_picture = item['achievements_picture']
                    one_steam.save()
        return item